package io.renren.modules.sys.back.service.impl;

import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

import io.renren.modules.sys.back.entity.BtCategroyManageEntity;


import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import org.jsoup.select.Elements;

import org.springframework.stereotype.Service;

import java.util.*;
import java.util.regex.Pattern;

public class SimpleCrawlerImpl extends WebCrawler {
   /* @Autowired
    private  BtCategroyManageService btCategroyManageDao ;*/
    /**
     * 正则表达式匹配指定的后缀文件
     */
    private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|gif|jpg" + "|png|mp3|mp3|zip|gz))$");

    /**
     * 这个方法主要是决定哪些url我们需要抓取，返回true表示是我们需要的，返回false表示不是我们需要的Url
     * 第一个参数referringPage封装了当前爬取的页面信息 第二个参数url封装了当前爬取的页面url信息
     * 在这个例子中，我们指定爬虫忽略具有css，js，git，...扩展名的url，只接受以“http://www.ics.uci.edu/”开头的url。
     * 在这种情况下，我们不需要referringPage参数来做出决定。
     */
    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String href = url.getURL().toLowerCase();
        boolean t = !FILTERS.matcher(href).matches() && href.startsWith("https://www.xianjichina.com/");
        return t;
    }

    /**
     * This function is called when a page is fetched and ready
     * to be processed by your program.
     */
    @Override
    public void visit(Page page) {

        String url = page.getWebURL().getURL();
       // System.out.println("URL: " + url);
        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            String html = htmlParseData.getHtml();
          //  Set<WebURL> links = htmlParseData.getOutgoingUrls();

            System.out.println("html;;;;;;;;;;;;;;;;;; " + html);
            Document doc = Jsoup.parse(html);
            Elements tagElement = doc.getElementsByClass("bread");
            Document doc2 = Jsoup.parse(tagElement.toString());
            Elements subdoc = doc2.getElementsByTag("a");
            CrawlController cc= this.getMyController();
            Map<String,Object> ccd= new HashMap<String,Object>();
            BtCategroyManageEntity msbean =null;
            if(subdoc.size()>2){

                for(int i=2;i<subdoc.size();i++){
                    String te =  subdoc.get(i).html();
                    if(i==2) {
                        msbean = new BtCategroyManageEntity();
                        msbean.setName(te);
                        msbean.setPid(0L);
                        msbean.setDeepnum(1);
                    }
                }
                cc.setCustomData(ccd);
            }
            Elements tagElementfl = doc.getElementsByClass("bk-listfltop");
            Document docfl = Jsoup.parse(tagElementfl.toString());
            Elements subdocfl = docfl.getElementsByTag("a");
            if(subdocfl.size()>1){
                List<BtCategroyManageEntity> subfl= new ArrayList<>();
                BtCategroyManageEntity mssbean;
                for(int i=1;i<subdocfl.size();i++){
                    String te1 =  subdocfl.get(i).html();
                    mssbean = new BtCategroyManageEntity();
                    mssbean.setName(te1);
                    mssbean.setPid(msbean.getId());
                    mssbean.setDeepnum(1);
                    subfl.add(mssbean);
                }
                msbean.setDeepnum(2);
                ccd.put("par",msbean);
                ccd.put("sublist",subfl);
                cc.setCustomData(ccd);
            }
          //  System.out.println("links;;;;;;;;;;;;;;;;;; " + links);
        }
    }
}
